package top.yudoge.dogeeditor.llm.util.tokenizer

import com.huaban.analysis.jieba.JiebaSegmenter
import org.springframework.stereotype.Component

class JiebaTokenizer : Tokenizer {

    private val tokenizer = JiebaSegmenter()

    override fun tokenize(input: String): List<String> {
        if (input.isBlank()) {
            return emptyList()
        }
        return tokenizer.sentenceProcess(input)
    }
}